{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/fanyixing/.local/python3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/home/fanyixing/.local/python3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "matchzoo version 2.1.0\n",
      "\n",
      "data loading ...\n",
      "data loaded as `train_pack_raw` `dev_pack_raw` `test_pack_raw`\n",
      "`ranking_task` initialized with metrics [normalized_discounted_cumulative_gain@3(0.0), normalized_discounted_cumulative_gain@5(0.0), mean_average_precision(0.0)]\n",
      "loading embedding ...\n",
      "embedding loaded as `glove_embedding`\n"
     ]
    }
   ],
   "source": [
    "%run init.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2118/2118 [00:00<00:00, 3724.89it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 18841/18841 [00:07<00:00, 2470.05it/s]\n",
      "Processing text_right with append: 100%|██████████| 18841/18841 [00:00<00:00, 445941.69it/s]\n",
      "Building FrequencyFilter from a datapack.: 100%|██████████| 18841/18841 [00:00<00:00, 70474.81it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 47716.48it/s]\n",
      "Processing text_left with extend: 100%|██████████| 2118/2118 [00:00<00:00, 342940.70it/s]\n",
      "Processing text_right with extend: 100%|██████████| 18841/18841 [00:00<00:00, 379138.05it/s]\n",
      "Building Vocabulary from a datapack.: 100%|██████████| 404415/404415 [00:00<00:00, 1847244.28it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2118/2118 [00:00<00:00, 4067.10it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 18841/18841 [00:07<00:00, 2516.30it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 67851.85it/s]\n",
      "Processing text_left with transform: 100%|██████████| 2118/2118 [00:00<00:00, 89496.74it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 75255.27it/s]\n",
      "Processing length_left with len: 100%|██████████| 2118/2118 [00:00<00:00, 356739.86it/s]\n",
      "Processing length_right with len: 100%|██████████| 18841/18841 [00:00<00:00, 400286.10it/s]\n",
      "Processing text_left with transform: 100%|██████████| 2118/2118 [00:00<00:00, 42205.68it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 36443.25it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 122/122 [00:00<00:00, 4076.75it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 1115/1115 [00:00<00:00, 2482.03it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 68513.29it/s]\n",
      "Processing text_left with transform: 100%|██████████| 122/122 [00:00<00:00, 74397.37it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 81470.46it/s]\n",
      "Processing length_left with len: 100%|██████████| 122/122 [00:00<00:00, 124654.10it/s]\n",
      "Processing length_right with len: 100%|██████████| 1115/1115 [00:00<00:00, 340342.69it/s]\n",
      "Processing text_left with transform: 100%|██████████| 122/122 [00:00<00:00, 36081.31it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 34092.08it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 237/237 [00:00<00:00, 3945.55it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2300/2300 [00:01<00:00, 2229.53it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 69715.12it/s]\n",
      "Processing text_left with transform: 100%|██████████| 237/237 [00:00<00:00, 91685.12it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 81219.95it/s]\n",
      "Processing length_left with len: 100%|██████████| 237/237 [00:00<00:00, 168597.36it/s]\n",
      "Processing length_right with len: 100%|██████████| 2300/2300 [00:00<00:00, 361686.38it/s]\n",
      "Processing text_left with transform: 100%|██████████| 237/237 [00:00<00:00, 36193.34it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 37573.42it/s]\n"
     ]
    }
   ],
   "source": [
    "preprocessor = mz.preprocessors.BasicPreprocessor(fixed_length_left=10, fixed_length_right=40, remove_stop_words=False)\n",
    "train_pack_processed = preprocessor.fit_transform(train_pack_raw)\n",
    "dev_pack_processed = preprocessor.transform(dev_pack_raw)\n",
    "test_pack_processed = preprocessor.transform(test_pack_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'filter_unit': <matchzoo.preprocessors.units.frequency_filter.FrequencyFilter at 0x7f4c81abd208>,\n",
       " 'vocab_unit': <matchzoo.preprocessors.units.vocabulary.Vocabulary at 0x7f4c7e0039b0>,\n",
       " 'vocab_size': 16674,\n",
       " 'embedding_input_dim': 16674,\n",
       " 'input_shapes': [(10,), (40,)]}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor.context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "model = mz.models.ConvKNRM()\n",
    "model.params.update(preprocessor.context)\n",
    "model.params['task'] = ranking_task\n",
    "model.params['embedding_output_dim'] = glove_embedding.output_dim\n",
    "model.params['embedding_trainable'] = True\n",
    "model.params['filters'] = 128 \n",
    "model.params['conv_activation_func'] = 'tanh' \n",
    "model.params['max_ngram'] = 3\n",
    "model.params['use_crossmatch'] = True \n",
    "model.params['kernel_num'] = 11\n",
    "model.params['sigma'] = 0.1\n",
    "model.params['exact_sigma'] = 0.001\n",
    "model.params['optimizer'] = 'adadelta'\n",
    "model.build()\n",
    "model.compile()\n",
    "#model.backend.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_matrix = glove_embedding.build_matrix(preprocessor.context['vocab_unit'].state['term_index'])\n",
    "model.load_embedding_matrix(embedding_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_x, pred_y = test_pack_processed.unpack()\n",
    "evaluate = mz.callbacks.EvaluateAllMetrics(model, x=pred_x, y=pred_y, batch_size=len(pred_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num batches: 255\n"
     ]
    }
   ],
   "source": [
    "train_generator = mz.DataGenerator(\n",
    "    train_pack_processed,\n",
    "    mode='pair',\n",
    "    num_dup=5,\n",
    "    num_neg=1,\n",
    "    batch_size=20\n",
    ")\n",
    "print('num batches:', len(train_generator))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "255/255 [==============================] - 39s 151ms/step - loss: 0.4776\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6046833551308459 - normalized_discounted_cumulative_gain@5(0.0): 0.6665959185697871 - mean_average_precision(0.0): 0.6129317182476187\n",
      "Epoch 2/30\n",
      "255/255 [==============================] - 42s 165ms/step - loss: 0.1211\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6152022692439065 - normalized_discounted_cumulative_gain@5(0.0): 0.6744661220937255 - mean_average_precision(0.0): 0.6343442709387471\n",
      "Epoch 3/30\n",
      "255/255 [==============================] - 43s 167ms/step - loss: 0.0620\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5548312155628421 - normalized_discounted_cumulative_gain@5(0.0): 0.6213976803862951 - mean_average_precision(0.0): 0.5710894912635418\n",
      "Epoch 4/30\n",
      "255/255 [==============================] - 43s 168ms/step - loss: 0.0417\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5954135056241098 - normalized_discounted_cumulative_gain@5(0.0): 0.6578139885107471 - mean_average_precision(0.0): 0.6111818707546555\n",
      "Epoch 5/30\n",
      "255/255 [==============================] - 42s 165ms/step - loss: 0.0356\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6028110339356899 - normalized_discounted_cumulative_gain@5(0.0): 0.6587081531783964 - mean_average_precision(0.0): 0.609672147851597\n",
      "Epoch 6/30\n",
      "255/255 [==============================] - 43s 167ms/step - loss: 0.0168\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6162374354841582 - normalized_discounted_cumulative_gain@5(0.0): 0.6730370811514368 - mean_average_precision(0.0): 0.6224984341793723\n",
      "Epoch 7/30\n",
      "255/255 [==============================] - 44s 171ms/step - loss: 0.0173\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6140235133098497 - normalized_discounted_cumulative_gain@5(0.0): 0.6702416124777638 - mean_average_precision(0.0): 0.6291591558219198\n",
      "Epoch 8/30\n",
      "255/255 [==============================] - 42s 167ms/step - loss: 0.0105\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6170925436547549 - normalized_discounted_cumulative_gain@5(0.0): 0.6721917042350951 - mean_average_precision(0.0): 0.6319963125925817\n",
      "Epoch 9/30\n",
      "255/255 [==============================] - 44s 173ms/step - loss: 0.0093\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6128277115636099 - normalized_discounted_cumulative_gain@5(0.0): 0.6723112242287127 - mean_average_precision(0.0): 0.6202358378889221\n",
      "Epoch 10/30\n",
      "255/255 [==============================] - 44s 172ms/step - loss: 0.0074\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6014563078514086 - normalized_discounted_cumulative_gain@5(0.0): 0.6534368691098246 - mean_average_precision(0.0): 0.6111100251131897\n",
      "Epoch 11/30\n",
      "255/255 [==============================] - 44s 174ms/step - loss: 0.0040\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6227886270617885 - normalized_discounted_cumulative_gain@5(0.0): 0.671961802050252 - mean_average_precision(0.0): 0.6294700258349492\n",
      "Epoch 12/30\n",
      "255/255 [==============================] - 43s 170ms/step - loss: 0.0049\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6090539823114218 - normalized_discounted_cumulative_gain@5(0.0): 0.6657954120017808 - mean_average_precision(0.0): 0.6204152162428167\n",
      "Epoch 13/30\n",
      "255/255 [==============================] - 44s 172ms/step - loss: 0.0045\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.605268576648384 - normalized_discounted_cumulative_gain@5(0.0): 0.6743645292160183 - mean_average_precision(0.0): 0.6225096268464414\n",
      "Epoch 14/30\n",
      "255/255 [==============================] - 44s 173ms/step - loss: 0.0056\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6094653731796733 - normalized_discounted_cumulative_gain@5(0.0): 0.6626866921233837 - mean_average_precision(0.0): 0.6167465211769009\n",
      "Epoch 15/30\n",
      "255/255 [==============================] - 44s 174ms/step - loss: 0.0020\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.636181345906811 - normalized_discounted_cumulative_gain@5(0.0): 0.6839535736459206 - mean_average_precision(0.0): 0.6396845610127441\n",
      "Epoch 16/30\n",
      "255/255 [==============================] - 44s 171ms/step - loss: 0.0020\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6242068770913558 - normalized_discounted_cumulative_gain@5(0.0): 0.6742098356127264 - mean_average_precision(0.0): 0.632390750330996\n",
      "Epoch 17/30\n",
      "255/255 [==============================] - 44s 173ms/step - loss: 9.0911e-04\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6305977929317165 - normalized_discounted_cumulative_gain@5(0.0): 0.6760885220490657 - mean_average_precision(0.0): 0.6345498605597264\n",
      "Epoch 18/30\n",
      "255/255 [==============================] - 44s 172ms/step - loss: 5.5907e-04\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6302281364338485 - normalized_discounted_cumulative_gain@5(0.0): 0.6801281668512741 - mean_average_precision(0.0): 0.6365815560435815\n",
      "Epoch 19/30\n",
      "255/255 [==============================] - 44s 174ms/step - loss: 0.0011\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6161750510905308 - normalized_discounted_cumulative_gain@5(0.0): 0.6828588425017452 - mean_average_precision(0.0): 0.6399974866560868\n",
      "Epoch 20/30\n",
      "255/255 [==============================] - 44s 173ms/step - loss: 5.9813e-04\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6192304771389455 - normalized_discounted_cumulative_gain@5(0.0): 0.6835492157788869 - mean_average_precision(0.0): 0.6339323846467368\n",
      "Epoch 21/30\n",
      "255/255 [==============================] - 43s 170ms/step - loss: 9.7819e-04\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6306940079499387 - normalized_discounted_cumulative_gain@5(0.0): 0.6811198627999581 - mean_average_precision(0.0): 0.6436540061956451\n",
      "Epoch 22/30\n",
      "255/255 [==============================] - 44s 171ms/step - loss: 7.8472e-04\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6246823923728032 - normalized_discounted_cumulative_gain@5(0.0): 0.6828731850846279 - mean_average_precision(0.0): 0.6295758423592317\n",
      "Epoch 27/30\n",
      "255/255 [==============================] - 44s 172ms/step - loss: 3.4582e-04\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6202967663352117 - normalized_discounted_cumulative_gain@5(0.0): 0.6718788743837789 - mean_average_precision(0.0): 0.6227787543285545\n",
      "Epoch 28/30\n",
      "255/255 [==============================] - 45s 177ms/step - loss: 0.0011\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6221372690184138 - normalized_discounted_cumulative_gain@5(0.0): 0.6775309597373549 - mean_average_precision(0.0): 0.6240701328331634\n",
      "Epoch 29/30\n",
      "255/255 [==============================] - 45s 177ms/step - loss: 9.9084e-04\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6193374900161305 - normalized_discounted_cumulative_gain@5(0.0): 0.6749664782289659 - mean_average_precision(0.0): 0.6316795270723043\n",
      "Epoch 30/30\n",
      "255/255 [==============================] - 45s 177ms/step - loss: 4.5966e-04\n",
      "Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6325513939822278 - normalized_discounted_cumulative_gain@5(0.0): 0.6801122164684699 - mean_average_precision(0.0): 0.6351265617879543\n"
     ]
    }
   ],
   "source": [
    "history = model.fit_generator(train_generator, epochs=30, callbacks=[evaluate], workers=30, use_multiprocessing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
